/**
 * 
 */
package cn.edu.bjtu.alex.rewrite.components;

import java.math.BigDecimal;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;


import com.google.common.collect.Maps;

import cn.edu.bjtu.alex.rewrite.Context;
import cn.edu.bjtu.alex.rewrite.interfaces.Term;
import cn.edu.bjtu.alex.rewrite.interfaces.TermFeatureable;
import cn.edu.bjtu.alex.rewrite.interfaces.impl.FeaturedTerm;
import cn.edu.bjtu.alex.rewrite.tools.MergeTermsPolicy;
import cn.edu.bjtu.alex.rewrite.tools.NamedThreadFactory;
import cn.edu.bjtu.alex.rewrite.tools.SimpleMergeTermsPolicy;
import cn.edu.bjtu.alex.rewrite.tools.SortUtils;
import cn.edu.bjtu.alex.rewrite.tools.SortUtils.Result;

/**
 * 进行特征选择,所有的操作都是基于Context类中的那几个数据结构进行操作的<br>
 * 这里同样用到了线程池
 * 
 * @author alex
 *
 */
public class SelectFeature extends BaseComponent{

	/**
	 * @param ctx
	 */
	public SelectFeature(Context ctx) {
		super(ctx);
	}

	protected void fitInternal() {
		Set<TermFeatureable> featureTerms = select(ctx);
		ctx.setFeaturedTerms(featureTerms);
	}
	
	// Map<label, Map<word, term>>
	private final Map<String, Map<String, TermFeatureable>> labeledTermsWithMeasures = Maps.newHashMap();
	// Map<label, ChiCalculator>
	private final Map<String, ChiCalculator> calculators = Maps.newHashMap();
	
	private ExecutorService executorService;
	private CountDownLatch latch;
	private MergeTermsPolicy mergeTermsPolicy;
	private float keptTermsPercent = 0.3f;
	
	public Set<TermFeatureable> select(Context context) {
		this.mergeTermsPolicy = new SimpleMergeTermsPolicy(this);
		//ctx.labelCount()是通过termTable.keySet().size();得到的文件分类数
		//而ctx.labels() 则是通过 Collections.unmodifiableList(labels);也就是ctx中的那个labels (List<String>)得到的,
		//原来的写法一个类中如果没有文件,也就是文件夹是空,并且线程中如果没有把labels中的对应的空类标的移除,就会造成数据不一致.就NPE了
		//现在给context加了removeLabel这个同步方法,将它移除之后数据就一致了.
		int labelCnt = ctx.labelCount();
		LOG.info("Initialize latch: labelCnt=" + labelCnt);
		latch = new CountDownLatch(labelCnt);
		executorService = Executors.newCachedThreadPool(new NamedThreadFactory("CHI"));
		try {
			for(String label : ctx.labels()) {
				labeledTermsWithMeasures.put(label, Maps.<String,TermFeatureable>newHashMap());
				calculators.put(label, new ChiCalculator(label, labeledTermsWithMeasures.get(label)));
				executorService.execute(calculators.get(label));
			}
		} finally {
			try {
				latch.await();
			} catch (InterruptedException e) { }
			LOG.info("Shutdown executor service: " + executorService);
			executorService.shutdown();
		}
		
		stat();
		
		// merge CHI terms for all labels
		return mergeTermsPolicy.merge(context, labeledTermsWithMeasures);
	}
	
	private void stat() {
		Iterator<Entry<String, Map<String, TermFeatureable>>> iter = labeledTermsWithMeasures.entrySet().iterator();
		while(iter.hasNext()) {
			Entry<String, Map<String, TermFeatureable>> entry = iter.next();
			String label = entry.getKey();
			LOG.info("STAT: label=" + label + ", featuredTerms=" + entry.getValue().size());
		}
	}

	private final class ChiCalculator extends Thread {
		
		private final String label;
		private final Map<String, TermFeatureable> terms;
		private Result sortedResult;
		public ChiCalculator(String label, Map<String, TermFeatureable> terms) {
			this.label = label;
			this.terms = terms;
		}
		@Override
		public void run() {
			try {
				LOG.info("Calc CHI: label=" + label);
				processSingleLabelledData();
				LOG.info("CHI calculated: label=" + label + ", termsCount=" + terms.size());
				
				LOG.info("Sort CHI terms for: label=" + label);
				Date start = new Date();
				int topN = (int) (terms.size() * keptTermsPercent);
				LOG.info("Terms selection for sort: topN=" + topN);
				sortedResult = sort(terms, topN);
				Date finish = new Date();
				LOG.info("CHI terms sorted: label=" + label + ", timeTaken=" + (finish.getTime() - start.getTime()) + "(ms)");
			} finally {
				latch.countDown();
			}
		}
		
		private Result sort(Map<String, TermFeatureable> terms, int topN) {
			SortUtils sorter = new SortUtils(terms, true, Math.min(topN, terms.size()));
			Result result = sorter.heapSort();
			return result;
		}
		
		private void processSingleLabelledData() {
			
			////  Map<word ,Map<label, Set<doc>>>
			Iterator<Entry<String, Map<String, Set<String>>>> iter = ctx.invertedTableIterator();
			while(iter.hasNext()) {
				Entry<String, Map<String, Set<String>>> entry = iter.next();
				String word = entry.getKey();
				Map<String, Set<String>> labelledDocs = entry.getValue();
				
				// A: doc count containing the word in this label
				int docCountContainingWordInLabel = 0;
				if(labelledDocs.get(label) != null) {
					docCountContainingWordInLabel = labelledDocs.get(label).size();
				}
				
				// B: doc count containing the word not in this label
				int docCountContainingWordNotInLabel = 0;
				Iterator<Entry<String, Set<String>>> labelledIter = labelledDocs.entrySet().iterator();
				while(labelledIter.hasNext()) {
					Entry<String, Set<String>> labelledEntry = labelledIter.next();
					String tmpLabel = labelledEntry.getKey();
					if(!label.equals(tmpLabel)) {
						docCountContainingWordNotInLabel += labelledEntry.getValue().size();
					}
				}
				
				// C: doc count not containing the word in this label
				int docCountNotContainingWordInLabel = computeDocCountNotContainingWordInLabel(word, label);
				
				// D: doc count not containing the word not in this label
				int docCountNotContainingWordNotInLabel = computeDocCountNotContainingWordNotInLabel(word, label);
				
				// compute CHI value
				double N = ctx.totalDocCount();
				double A = docCountContainingWordInLabel;
				double B = docCountContainingWordNotInLabel;
				double C = docCountNotContainingWordInLabel;
				double D = docCountNotContainingWordNotInLabel;
				double temp = (A*D-B*C);
				double chi = N*temp*temp / ((A+C)*(A+B)*(B+D)*(C+D));
//				if(chi < 0){
//					System.out.println("N = " + N + " , A= " + A + ", B= " + B + " , C= " + C + " , D = " + D + "   word: " + word + " , CHI= " + chi);
//				}
				TermFeatureable term = new FeaturedTerm(word);
				term.setMeasureValue(chi);
				terms.put(word, term);
			}
		}

		private int computeDocCountNotContainingWordInLabel(String word, String label) {
			int count = 0;
			Iterator<Entry<String,Map<String,Map<String,Term>>>> iter = ctx.termTableIterator();
			while(iter.hasNext()) {
				Entry<String,Map<String,Map<String,Term>>> entry = iter.next();
				String tmpLabel = entry.getKey();
				// in this label
				if(tmpLabel.equals(label)) {
					Map<String, Map<String, Term>> labelledDocs = entry.getValue();
					for(Entry<String, Map<String, Term>> docEntry : labelledDocs.entrySet()) {
						// not containing this word
						if(!docEntry.getValue().containsKey(word)) {
							++count;
						}
					}
					break;
				}
			}
			return count;
		}
		
		private int computeDocCountNotContainingWordNotInLabel(String word, String label) {
			int count = 0;
			Iterator<Entry<String,Map<String,Map<String,Term>>>> iter = ctx.termTableIterator();
			while(iter.hasNext()) {
				Entry<String,Map<String,Map<String,Term>>> entry = iter.next();
				String tmpLabel = entry.getKey();
				// not in this label
				if(!tmpLabel.equals(label)) {
					Map<String, Map<String, Term>> labelledDocs = entry.getValue();
					for(Entry<String, Map<String, Term>> docEntry : labelledDocs.entrySet()) {
						// not containing this word
						if(!docEntry.getValue().containsKey(word)) {
							++count;
						}
					}
				}
			}
			return count;
		}
	}
	
	public Result getSortedResult(String label) {
		return calculators.get(label).sortedResult;
	}

	/* (non-Javadoc)
	 * @see cn.edu.bjtu.alex.rewrite.components.BaseComponent#transformInternal()
	 */
	@Override
	protected void transformInternal() {
		// TODO Auto-generated method stub
		
	}
	
//	public void load(Context context) {
//		BufferedReader reader = null;
//		FileInputStream fis = null;
//		try {
//			fis = new FileInputStream(ctx.getFeatureTermVectorFile());
//			reader = new BufferedReader(new InputStreamReader(fis, context.getCharset()));
//			String line = null;
//			Set<TermFeatureable> terms = Sets.newHashSet();
//			while((line = reader.readLine()) != null) {
//				line = line.trim();
//				if(!line.isEmpty()) {
//					String[] aWord = line.split("\\s+");
//					if(aWord.length == 2) {
//						String word = aWord[0];
//						int wordId = Integer.parseInt(aWord[1]);
//						FeaturedTerm term = new FeaturedTerm(word);
//						term.setId(wordId);
//						terms.add(term);
//						LOG.info("Load CHI term: word=" + word + ", wordId=" + wordId);
//					}
//				}
//			}
//			context.getVectorMetadata().setFeaturedTerms(terms);
//		} catch (UnsupportedEncodingException e) {
//			e.printStackTrace();
//		} catch (IOException e) {
//			e.printStackTrace();
//		} finally {
//			FileUtils.closeQuietly(fis, reader);
//		}		
//	}


}
